package de.dopichaj.labrador.tools;


import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.lang.StringUtils;

import de.dopichaj.labrador.index.IndexVersionException;
import de.dopichaj.labrador.index.backend.Dictionary;
import de.dopichaj.labrador.index.own.OwnFileSystemIndex;
import de.dopichaj.labrador.search.NEXISearcher;
import de.dopichaj.labrador.search.QueryParseException;
import de.dopichaj.labrador.search.SearchException;
import de.dopichaj.labrador.search.hit.DocHitMap;
import de.dopichaj.labrador.search.hit.Hit;


public final class CoOccurrenceFinder {

    private static final double THRESHOLD_LOW = 0.7;
    private static final double THRESHOLD_HIGH = 0.9;
    private static final int MAX_ITEMS_SIZE = 25000;
    
    public static void main(String[] args) throws IOException, IndexVersionException, SearchException, QueryParseException {

        for (final String arg : args) {
            printCoOccurrences(new File(arg));
        }
    }

    private static void printCoOccurrences(final File file) throws IOException, IndexVersionException, SearchException, QueryParseException {
        final OwnFileSystemIndex index = new OwnFileSystemIndex(file);
        index.getConfiguration().setString("similarityFactory", "de.dopichaj.labrador.search.own.CoordinateLevelSimilarityFactory");
        
        System.err.println(file);
        // create a list of all terms in the suitable range
        final Dictionary dictionary = new Dictionary();
        dictionary.read(index.getDictionaryFile());
        final int documentCount = index.getDocumentCount();
        Set<BitSet> candidates = new HashSet<BitSet>();
        int bitPos = 0;
        final List<String> wordList = new ArrayList<String>();
        for (final String word : dictionary) {
            
            if (word.matches("^\\d*$")) {
                continue;
            }
            final int count = dictionary.getTerm(word).getDocumentFrequency();
            final double frequency = (double)count / documentCount;
            if (isInRange(frequency)) {
                wordList.add(word);
                final BitSet set = new BitSet(bitPos);
                set.set(bitPos);
                candidates.add(set);
                bitPos++;
            }
        }
        
        final NEXISearcher searcher = index.getNEXISearcher();
        Collection<BitSet> matches = new ArrayList<BitSet>();
        int size = 1;
        do {
            System.err.println(size + ": " + candidates.size());
            // for each item in the candidate set
            for (final BitSet item : candidates) {
                
                // de.dopichaj.labrador.search for all terms using coordinate-level matching
                final StringBuilder query = new StringBuilder();
                for (int i = item.nextSetBit(0); i >= 0; i = item.nextSetBit(i+1)) {
                    query.append(wordList.get(i)).append(" ");
                }
                final DocHitMap result = searcher.nexiSearch(query.toString());

                // determine the number of matches that contain all terms
                int coCount = 0;
                for (final Hit hit : result.getHits()) {
                    if (hit.getScore() == item.cardinality()) {
                        coCount++;
                    }
                }

                // if the frequency is in range
                final double coFreq = (double)coCount / documentCount;
                if (isInRange(coFreq)) {

                    // add the item to the match list
                    matches.add(item);
                }
            }
            
            // determine the next candidate set
            candidates = getNextCandidates(matches, size);
            size++;

        // while the next candidate set is not empty and the maximum size has not been reached
        } while (candidates.size() > 0);

        
        // print the matches
        for (final BitSet match : matches) {
            final List<String> s = new ArrayList<String>();
            for (int i = match.nextSetBit(0); i >= 0; i = match.nextSetBit(i+1)) {
                s.add(wordList.get(i));
            }
            Collections.sort(s);
            System.out.println(file + "\t" + s.size() + "\t" + StringUtils.join(s.iterator(), ";"));
        }
        
        searcher.close();
    }

    private static Set<BitSet> getNextCandidates(Collection<BitSet> matches, int size) {
        
        // create a list of all items that are large enough
        final List<BitSet> items = new ArrayList<BitSet>();
        for (final BitSet item : matches) {
            if (item.cardinality() >= size - 1) {
                items.add(item);
            }
        }
        System.err.println(items.size());
        if (items.size() >= MAX_ITEMS_SIZE) {
            System.err.println("Cowardly giving up");
            return Collections.emptySet();
        }
        
        // for each item in the list
        final Set<BitSet> result = new HashSet<BitSet>();
        for (int i = 0; i < items.size(); i++) {
            final BitSet set1 = items.get(i);
            final int low1 = set1.nextSetBit(0);
            final int high1 = set1.length();
            
            // for each item that follows it in the list
            for (int j = i+1; j < items.size(); j++) {
                final BitSet set2 = items.get(j);
                final int low2 = set2.nextSetBit(0);
                final int high2 = set2.length();
                
                // if there can be any intersection
                final int commonInterval = Math.min(high1, high2) - Math.max(low1, low2);
                if (size - 1 == 0 || (commonInterval >= size - 1 && set1.intersects(set2))) {
                
                    // if the intersection has the correct size
                    final BitSet intersection = new BitSet(set1.length());
                    intersection.or(set1);
                    intersection.and(set2);
                    if (intersection.cardinality() == size - 1) {
                    
                        // add the union to the result
                        final BitSet union = intersection;
                        union.or(set1);
                        union.or(set2);
                        result.add(union);
                    }
                }
            }
        }
        
        return result;
    }

    private static boolean isInRange(final double frequency) {
        return THRESHOLD_LOW <= frequency && frequency <= THRESHOLD_HIGH;
    }
}
/*
Copyright (c) 2007 Philipp Dopichaj

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/